{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensorflow的Layer基础知识介绍\n",
    "\n",
    "Layer是构成Model的基本单元，简单理解成一个数据处理函数：\n",
    "* Layer可以输入一个数据tensor，经过代码处理，输出一个数据Tensor\n",
    "* Layer中有参数Weights，Model训练的目标，就是提供数据(features、label），优化Model中的Layer的Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras import layers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Layer就像函数一样调用"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dense即全连接层\n",
    "layer = layers.Dense(3, activation='relu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(5, 6), dtype=float32, numpy=\n",
       "array([[0.33835745, 0.4028859 , 0.23511803, 0.21115911, 0.62988913,\n",
       "        0.15494311],\n",
       "       [0.18937635, 0.2366414 , 0.5940547 , 0.07008147, 0.7589654 ,\n",
       "        0.89733934],\n",
       "       [0.9675964 , 0.85580623, 0.91624475, 0.75455165, 0.7629715 ,\n",
       "        0.1385355 ],\n",
       "       [0.15827405, 0.9905658 , 0.9414414 , 0.7063625 , 0.31134558,\n",
       "        0.9627578 ],\n",
       "       [0.1599555 , 0.0049727 , 0.21972692, 0.76499987, 0.15703976,\n",
       "        0.22014487]], dtype=float32)>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 输入数据为一个tensor\n",
    "inputs = tf.random.uniform(shape=(5, 6))\n",
    "inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(5, 3), dtype=float32, numpy=\n",
       "array([[0.7080624 , 0.        , 0.        ],\n",
       "       [0.42272323, 0.4542595 , 0.        ],\n",
       "       [1.2965715 , 0.        , 0.        ],\n",
       "       [0.45373344, 0.        , 0.        ],\n",
       "       [0.5377994 , 0.        , 0.        ]], dtype=float32)>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 执行函数变换，注意输入输出的形状也变化了\n",
    "layer(inputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Layer本身常用的方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'dense/kernel:0' shape=(6, 3) dtype=float32, numpy=\n",
       " array([[ 0.28796923, -0.13785005, -0.27724725],\n",
       "        [ 0.11302459, -0.7230098 , -0.6558574 ],\n",
       "        [-0.11057937, -0.6325079 , -0.57528687],\n",
       "        [ 0.58731425, -0.5676677 , -0.7046772 ],\n",
       "        [ 0.80964005,  0.70802915, -0.5309555 ],\n",
       "        [-0.27694768,  0.59020793,  0.44910252]], dtype=float32)>,\n",
       " <tf.Variable 'dense/bias:0' shape=(3,) dtype=float32, numpy=array([0., 0., 0.], dtype=float32)>]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 虽然Layer可以像函数一样调用处理输入得到输出，但是它本身维护了可以更新的Weights参数\n",
    "layer.weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'dense/kernel:0' shape=(6, 3) dtype=float32, numpy=\n",
       " array([[ 0.28796923, -0.13785005, -0.27724725],\n",
       "        [ 0.11302459, -0.7230098 , -0.6558574 ],\n",
       "        [-0.11057937, -0.6325079 , -0.57528687],\n",
       "        [ 0.58731425, -0.5676677 , -0.7046772 ],\n",
       "        [ 0.80964005,  0.70802915, -0.5309555 ],\n",
       "        [-0.27694768,  0.59020793,  0.44910252]], dtype=float32)>,\n",
       " <tf.Variable 'dense/bias:0' shape=(3,) dtype=float32, numpy=array([0., 0., 0.], dtype=float32)>]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# trainable_weights会在训练期间，通过梯度下降进行更新\n",
    "layer.trainable_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[ 0.28796923, -0.13785005, -0.27724725],\n",
       "        [ 0.11302459, -0.7230098 , -0.6558574 ],\n",
       "        [-0.11057937, -0.6325079 , -0.57528687],\n",
       "        [ 0.58731425, -0.5676677 , -0.7046772 ],\n",
       "        [ 0.80964005,  0.70802915, -0.5309555 ],\n",
       "        [-0.27694768,  0.59020793,  0.44910252]], dtype=float32),\n",
       " array([0., 0., 0.], dtype=float32)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 以含有Numpy矩阵的列表形式返回层的权重\n",
    "layer.get_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'dense/kernel:0' shape=(6, 3) dtype=float32, numpy=\n",
       "array([[ 0.28796923, -0.13785005, -0.27724725],\n",
       "       [ 0.11302459, -0.7230098 , -0.6558574 ],\n",
       "       [-0.11057937, -0.6325079 , -0.57528687],\n",
       "       [ 0.58731425, -0.5676677 , -0.7046772 ],\n",
       "       [ 0.80964005,  0.70802915, -0.5309555 ],\n",
       "       [-0.27694768,  0.59020793,  0.44910252]], dtype=float32)>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer.kernel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'dense/bias:0' shape=(3,) dtype=float32, numpy=array([0., 0., 0.], dtype=float32)>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer.bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'name': 'dense',\n",
       " 'trainable': True,\n",
       " 'dtype': 'float32',\n",
       " 'units': 3,\n",
       " 'activation': 'relu',\n",
       " 'use_bias': True,\n",
       " 'kernel_initializer': {'class_name': 'GlorotUniform',\n",
       "  'config': {'seed': None}},\n",
       " 'bias_initializer': {'class_name': 'Zeros', 'config': {}},\n",
       " 'kernel_regularizer': None,\n",
       " 'bias_regularizer': None,\n",
       " 'activity_regularizer': None,\n",
       " 'kernel_constraint': None,\n",
       " 'bias_constraint': None}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 包含层配置的字典\n",
    "layer.get_config()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. 自己实现Dense全连接层"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dense 实现以下操作： output = activation(dot(input, kernel) + bias)：\n",
    "* kernel 是由网络层创建的权值矩阵\n",
    "* bias 是其创建的偏置向量\n",
    "* activation 是按逐个元素计算的激活函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleDense(tf.keras.layers.Layer):\n",
    "    \"\"\"要实现一个自己的Layer，首先继承tf.keras.Layer\"\"\"\n",
    "    \n",
    "    def __init__(self, units=32, activation=tf.keras.activations.relu):\n",
    "        \"\"\"在这里实现跟输入数据无关的初始化，units是输出单元个数\"\"\"\n",
    "        super(SimpleDense, self).__init__()\n",
    "        self.units = units\n",
    "        self.activation = activation\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        \"\"\"这里你已经知道了输入shape，做一些其它初始化\"\"\"\n",
    "        w_init = tf.random_normal_initializer()\n",
    "        self.w = tf.Variable(initial_value=w_init(shape=(input_shape[-1], self.units), dtype='float32'), trainable=True)\n",
    "        \n",
    "        b_init = tf.zeros_initializer()\n",
    "        self.b = tf.Variable(initial_value=b_init(shape=(self.units,), dtype='float32'), trainable=True)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        \"\"\"在这里执行真正的计算，m*n @ n*k = m*k\"\"\"\n",
    "        return self.activation(tf.matmul(inputs, self.w) + self.b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "simpleDense = SimpleDense(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(5, 6), dtype=float32, numpy=\n",
       "array([[0.84321594, 0.94971406, 0.00291455, 0.15217948, 0.18815064,\n",
       "        0.7553489 ],\n",
       "       [0.45222616, 0.8940605 , 0.66049397, 0.7730479 , 0.25303853,\n",
       "        0.08674979],\n",
       "       [0.04443181, 0.48009157, 0.58369625, 0.7755337 , 0.58347166,\n",
       "        0.21425164],\n",
       "       [0.6621847 , 0.75676584, 0.24291241, 0.1357758 , 0.78865397,\n",
       "        0.6802577 ],\n",
       "       [0.7083311 , 0.48510265, 0.24692857, 0.2819258 , 0.8745481 ,\n",
       "        0.12379682]], dtype=float32)>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 输入数据\n",
    "inputs = tf.random.uniform(shape=(5, 6))\n",
    "inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(5, 3), dtype=float32, numpy=\n",
       "array([[0.04327397, 0.04678761, 0.09832035],\n",
       "       [0.07879104, 0.1084666 , 0.00513381],\n",
       "       [0.04440111, 0.08707336, 0.        ],\n",
       "       [0.05931166, 0.06264593, 0.0342182 ],\n",
       "       [0.08149881, 0.08866378, 0.        ]], dtype=float32)>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simpleDense(inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
